{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c1gUG3OCJ5GS"
      },
      "source": [
        "# **Pendulum-v1 Example in ElegantRL-HelloWorld**\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DbamGVHC3AeW"
      },
      "source": [
        "# **Part 1: Install ElegantRL**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U35bhkUqOqbS",
        "outputId": "1536fb29-9e4c-4598-f26f-a907c591a8ba"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting git+https://github.com/AI4Finance-LLC/ElegantRL.git\n",
            "  Cloning https://github.com/AI4Finance-LLC/ElegantRL.git to /tmp/pip-req-build-1hyi8kfj\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/AI4Finance-LLC/ElegantRL.git /tmp/pip-req-build-1hyi8kfj\n",
            "  Resolved https://github.com/AI4Finance-LLC/ElegantRL.git to commit 6f7096c9f825d529b90cf8cba4fc75929af979ed\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from ElegantRL==0.3.10) (2.4.1+cu121)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from ElegantRL==0.3.10) (1.26.4)\n",
            "Collecting gymnasium (from ElegantRL==0.3.10)\n",
            "  Downloading gymnasium-1.0.0-py3-none-any.whl.metadata (9.5 kB)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from ElegantRL==0.3.10) (3.7.1)\n",
            "Requirement already satisfied: cloudpickle>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from gymnasium->ElegantRL==0.3.10) (2.2.1)\n",
            "Requirement already satisfied: typing-extensions>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from gymnasium->ElegantRL==0.3.10) (4.12.2)\n",
            "Collecting farama-notifications>=0.0.1 (from gymnasium->ElegantRL==0.3.10)\n",
            "  Downloading Farama_Notifications-0.0.4-py3-none-any.whl.metadata (558 bytes)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->ElegantRL==0.3.10) (1.3.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->ElegantRL==0.3.10) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->ElegantRL==0.3.10) (4.54.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->ElegantRL==0.3.10) (1.4.7)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->ElegantRL==0.3.10) (24.1)\n",
            "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->ElegantRL==0.3.10) (10.4.0)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->ElegantRL==0.3.10) (3.1.4)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib->ElegantRL==0.3.10) (2.8.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch->ElegantRL==0.3.10) (3.16.1)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch->ElegantRL==0.3.10) (1.13.3)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch->ElegantRL==0.3.10) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch->ElegantRL==0.3.10) (3.1.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch->ElegantRL==0.3.10) (2024.6.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib->ElegantRL==0.3.10) (1.16.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch->ElegantRL==0.3.10) (2.1.5)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch->ElegantRL==0.3.10) (1.3.0)\n",
            "Downloading gymnasium-1.0.0-py3-none-any.whl (958 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m958.1/958.1 kB\u001b[0m \u001b[31m10.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading Farama_Notifications-0.0.4-py3-none-any.whl (2.5 kB)\n",
            "Building wheels for collected packages: ElegantRL\n",
            "  Building wheel for ElegantRL (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for ElegantRL: filename=ElegantRL-0.3.10-py3-none-any.whl size=385443 sha256=d56ab75d104550140310837a32d2d7234eebf76afc3846174eb25693d7e48db6\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-gx2sgm52/wheels/85/a7/e7/37369f52de5c5e77ba80ec8d0dd898ef99fbb23707e77a3958\n",
            "Successfully built ElegantRL\n",
            "Installing collected packages: farama-notifications, gymnasium, ElegantRL\n",
            "Successfully installed ElegantRL-0.3.10 farama-notifications-0.0.4 gymnasium-1.0.0\n"
          ]
        }
      ],
      "source": [
        "# install elegantrl library\n",
        "!pip install git+https://github.com/AI4Finance-LLC/ElegantRL.git"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3n8zcgcn14uq"
      },
      "source": [
        "# **Part 2: Specify Environment and Agent**\n",
        "\n",
        "*   **agent**: chooses a agent (DRL algorithm) from a set of agents in the [directory](https://github.com/AI4Finance-Foundation/ElegantRL/tree/master/elegantrl/agents).\n",
        "*   **env**: creates an environment for your agent.\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import gymnasium as gym"
      ],
      "metadata": {
        "id": "1eyRQkk9pkBf"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "E03f6cTeajK4"
      },
      "outputs": [],
      "source": [
        "from elegantrl.train.config import Config\n",
        "from elegantrl.agents.AgentSAC import AgentSAC\n",
        "\n",
        "env = gym.make\n",
        "\n",
        "env_args = {\n",
        "    \"id\": \"Pendulum-v1\",\n",
        "    \"env_name\": \"Pendulum-v1\",\n",
        "    \"num_envs\": 1,\n",
        "    \"max_step\": 1000,\n",
        "    \"state_dim\": 3,\n",
        "    \"action_dim\": 1,\n",
        "    \"if_discrete\": False,\n",
        "    \"reward_scale\": 2**-1,\n",
        "    \"gpu_id\": 0, # if you have GPU\n",
        "}\n",
        "args = Config(AgentSAC, env_class=env, env_args=env_args)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rcFcUkwfzHLE"
      },
      "source": [
        "# **Part 3: Specify hyper-parameters**\n",
        "A list of hyper-parameters is available [here](https://elegantrl.readthedocs.io/en/latest/api/config.html)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "9WCAcmIfzGyE"
      },
      "outputs": [],
      "source": [
        "args.max_step = 1000\n",
        "args.reward_scale = 2**-1  # RewardRange: -1800 < -200 < -50 < 0\n",
        "args.gamma = 0.97\n",
        "args.target_step = args.max_step\n",
        "args.eval_times = 2**3\n",
        "args.gpu_id = 0"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z1j5kLHF2dhJ"
      },
      "source": [
        "# **Part 4: Train and Evaluate the Agent**\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KGOPSD6da23k",
        "outputId": "fd200e35-217c-41d6-b01c-e4a8a535840c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "| train_agent_multiprocessing() with GPU_ID 0\n",
            "| Arguments Remove cwd: ./Pendulum-v1_SAC_0\n"
          ]
        }
      ],
      "source": [
        "from elegantrl.train.run import train_agent\n",
        "\n",
        "train_agent(args)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JPXOxLSqh5cP"
      },
      "source": [
        "Understanding the above results::\n",
        "*   **Step**: the total training steps.\n",
        "*  **MaxR**: the maximum reward.\n",
        "*   **avgR**: the average of the rewards.\n",
        "*   **stdR**: the standard deviation of the rewards.\n",
        "*   **objA**: the objective function value of Actor Network (Policy Network).\n",
        "*   **objC**: the objective function value (Q-value)  of Critic Network (Value Network)."
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "quickstart_Pendulum-v1.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.6"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}